{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "from selenium.webdriver import ActionChains\n",
    "import time, random\n",
    "import requests\n",
    "import re\n",
    "import json\n",
    "import pickle\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def get_proxy_ip():\n",
    "#     try:\n",
    "#         zhima_ip = r'http://webapi.http.zhimacangku.com/getip?num=1&type=1&pro=&city=0&yys=0&port=11&pack=34560&ts=0&ys=0&cs=0&lb=1&sb=0&pb=4&mr=1&regions='\n",
    "#         proxy = requests.get(zhima_ip).text[:-2]\n",
    "#         time.sleep(1.5)\n",
    "#         prox = {\n",
    "#             'https': 'https://{}'.format(proxy),\n",
    "#             'http': 'http://{}'.format(proxy)}\n",
    "#         res = requests.get('http://2018.ip138.com/ic.asp', proxies=prox, timeout=10)\n",
    "#         if res.status_code == 200:\n",
    "#             return proxy.split(':')\n",
    "#         else:\n",
    "#             return get_proxy_ip()\n",
    "#     except:\n",
    "#         return get_proxy_ip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "# proxy_url, port = get_proxy_ip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "# user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.3538.102 Safari/537.36'\n",
    "user_agent = \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36\"\n",
    "# user_agent = 'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; .NET4.0C; .NET4.0E; .NET CLR 2.0.50727; .NET CLR 3.0.30729; .NET CLR 3.5.30729; InfoPath.3; rv:11.0) like Gecko'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "fp = webdriver.FirefoxProfile()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "fp.set_preference(\"general.useragent.override\", user_agent)\n",
    "# fp.set_preference('network.proxy.type', 1)\n",
    "# fp.set_preference('network.proxy.http', proxy_url)\n",
    "# fp.set_preference('network.proxy.http_port', int(port))\n",
    "# fp.set_preference('network.proxy.ssl', proxy_url)\n",
    "# fp.set_preference('network.proxy.ssl_port', int(port))\n",
    "fp.update_preferences()\n",
    "driver = webdriver.Firefox(firefox_profile=fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "js1 = '''() =>{\n",
    "    \n",
    "           Object.defineProperties(navigator,{\n",
    "             webdriver:{\n",
    "               get: () => false\n",
    "             }\n",
    "           })\n",
    "        }'''\n",
    " \n",
    "js2 = '''() => {\n",
    "        alert (\n",
    "            window.navigator.webdriver\n",
    "        )\n",
    "    }'''\n",
    " \n",
    "js3 = '''() => {\n",
    "        window.navigator.chrome = {\n",
    "    runtime: {},\n",
    "    // etc.\n",
    "  };\n",
    "    }'''\n",
    " \n",
    "js4 = '''() =>{\n",
    "Object.defineProperty(navigator, 'languages', {\n",
    "      get: () => ['en-US', 'en']\n",
    "    });\n",
    "        }'''\n",
    " \n",
    "js5 = '''() =>{\n",
    "Object.defineProperty(navigator, 'plugins', {\n",
    "    get: () => [1, 2, 3, 4, 5,6],\n",
    "  });\n",
    "        }'''\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "def set_navigator():\n",
    "    driver.execute_script(js1)\n",
    "    driver.execute_script(js2)\n",
    "    driver.execute_script(js3)\n",
    "    driver.execute_script(js4)\n",
    "    driver.execute_script(js5)\n",
    "    print('navigator set successfully')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "navigator set successfully\n"
     ]
    }
   ],
   "source": [
    "set_navigator()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "login_url = r'https://strategy.tmall.com/welcome#/marketSeg/27647?_k=0zis0l'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.get(login_url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "navigator set successfully\n"
     ]
    }
   ],
   "source": [
    "driver.refresh()\n",
    "set_navigator()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "# driver.firefox_profile.update_preferences()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "# driver.firefox_profile.default_preferences['network.proxy.type'] = 1\n",
    "# driver.firefox_profile.default_preferences['network.proxy.http'] = proxy_url\n",
    "# driver.firefox_profile.default_preferences['network.proxy.http_port'] = int(port)\n",
    "# driver.firefox_profile.default_preferences['network.proxy.ssl'] = proxy_url\n",
    "# driver.firefox_profile.default_preferences['network.proxy.ssl_port'] = port\n",
    "# driver.firefox_profile.update_preferences()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36'"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "iframe = driver.find_element_by_id('tbLoginFrame')\n",
    "driver.switch_to.frame(iframe)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.find_element_by_css_selector('html.ks-webkit537.ks-webkit.ks-chrome70.ks-chrome body div#page2 div#content div#J_LoginBox.login-box.no-longlogin.module-quick div.bd div.quick-form div#J_QRCodeLogin.qrcode-login div.login-links a.forget-pwd.J_Quick2Static').click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.find_element_by_css_selector('#TPL_username_1').send_keys('森马官方旗舰店:欣月')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.find_element_by_css_selector('#TPL_password_1').send_keys('semir2019')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.find_element_by_css_selector('#J_SubmitStatic').click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# driver.find_element_by_css_selector('#TPL_password_1').send_keys('www.,194928.,com')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# action = ActionChains(driver)\n",
    "# source=driver.find_element_by_xpath(\"//*[@id='nc_1_n1t']/span\")#需要滑动的元素\n",
    "# action.click_and_hold(source).perform()\n",
    "# time.sleep(0.5)\n",
    "# action.move_by_offset(206,0)#需要滑动的坐标\n",
    "# time.sleep(1.2)\n",
    "# action.release().perform() #释放鼠标"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# script_refresh = 'javascript:noCaptcha.reset(1)'\n",
    "# driver.execute_script(script_refresh)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "cookie = driver.get_cookies()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "cookies_dict = {}\n",
    "for x_dict in cookie:\n",
    "    cookies_dict[x_dict['name']] = x_dict['value']\n",
    "for x_dict in cookie:\n",
    "    x_dict_new = {x:y for x, y in x_dict.items() if x != 'name' and x != 'value'}\n",
    "    cookies_dict.update(x_dict_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "cookies_dict2 = {x:str(y) for x, y in cookies_dict.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "home_url = 'https://strategy.tmall.com/#/marketSeg/28453'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "headers = {'Host': 'strategy.tmall.com',\n",
    "           'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36',\n",
    "           'Accept': '*/*',\n",
    "           'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',\n",
    "           'Accept-Encoding': 'gzip, deflate, br',\n",
    "           'Referer': 'https://strategy.tmall.com/',\n",
    "           'content-type': 'application/json',\n",
    "           'origin': 'https://strategy.tmall.com',\n",
    "           'Connection': 'keep-alive'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "response = requests.get(home_url, headers=headers, cookies=cookies_dict2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\r\\n<!DOCTYPE html>\\n<html lang=\"en\">\\n<head>\\n  <meta charset=\"UTF-8\">\\n  <meta name=\"viewport\" content=\"width=device-width, initial-scale=1.0\">\\n  <meta http-equiv=\"X-UA-Compatible\" content=\"ie=edge\">\\n  <link rel=\"shortcut icon\" href=\"//img.alicdn.com/tfs/TB1_5kWnKuSBuNjy1XcXXcYjFXa-32-32.png\">\\n  <meta name=\"_csrf\" token=\"57b7e373e6533\">\\n  <meta name=\"data-spm\" data-spm-protocol=\"i\" content=\"a2o2t\" />\\n  <meta name=\"aplus-ajax\" content=\"chksum\">\\n  <meta name=\"aplus-wating\" content=\"1\">\\n  <title>策略中心</title>\\n</head>\\n<body><script>\\nwith(document)with(body)with(insertBefore(createElement(\"script\"),firstChild))setAttribute(\"exparams\",\"category=&userid=2847965855&aplus&yunid=&57b7e373e6533&trid=0b0f57ec15558185882357821ea0a8&asid=AZ+CwKld6LtcCQgbPQAAAAAp0VZbSp1HPw==\",id=\"tb-beacon-aplus\",src=(location>\"https\"?\"//g\":\"//g\")+\".alicdn.com/alilog/mlog/aplus_v2.js\")\\n</script>\\n\\n  <div id=\"content_container\"></div>\\n  <script>\\n    window.baseConfig = {\\n      nick: \\'森马官方旗舰店:欣月\\',\\n      baseId: \\'4e843e3b-9c61-4064-89d7-2f04d8dfc143\\',\\n      accountName: \\'森马官方旗舰店:欣月\\',\\n      isOwner: false,\\n      smartId: 619,\\n      smartType: 0,\\n      isWLogin: false,\\n      brandId: 130259,\\n      brandName: \\'Semir/森马\\',\\n      logoutUrl: \\'//strategy.tmall.com/signOut\\',\\n      nextBiUrl: \\'//insight-engine.tmall.com/\\',\\n      env: \\'PROD\\',\\n      timestamp: 1555818589273,\\n\\t    policyCenterUrl: \\'http://smart.yushanfang.com/human/crowd-choose#/crowd-analysis\\',\\n      databankUrl: \\'//databank.tmall.com/\\',\\n      databankFactoryUrl: \\'https://paas-databank.tmall.com\\',\\n      goToFactoryUrl: \\'/goToFactory\\',\\n      assetsUrl: \\'//g.alicdn.com/base-tech-fe/saas-strategy-fe/\\',\\n      isDemo: false,\\n      departmentList: [{\"departmentId\":10619,\"departmentName\":\"电商部\",\"ownerAccountName\":null,\"bizType\":\"STRATEGY\",\"projectId\":179018,\"smartId\":619,\"selected\":false}],\\n      currentDepartmentId: 10619,\\n      empId: \\'\\',\\n      lastName: \\'\\',\\n      nickName: \\'\\',\\n      industryList: [],\\n      auth: [{\"code\":\"38\",\"status\":1},{\"code\":\"marketOverview\",\"status\":1},{\"code\":\"marketSeg\",\"status\":1},{\"code\":\"customCrowd\",\"status\":1},{\"code\":\"correlationAnalysis\",\"status\":1},{\"code\":\"consumerPathAnalysis\",\"status\":1},{\"code\":\"categoryAnalysis\",\"status\":1},{\"code\":\"brandGainAndLoss\",\"status\":1},{\"code\":\"penetrationTrack\",\"status\":1},{\"code\":\"productTrack\",\"status\":1},{\"code\":\"partHomeIndex\",\"status\":0},{\"code\":\"partLogo\",\"status\":0},{\"code\":\"brandGainAndLossMarketSeg\",\"status\":1},{\"code\":\"darenOptimal\",\"status\":1},{\"code\":\"znfqLife\",\"status\":0}],\\n      a: \\'\\',\\n      b: \\'\\',\\n      c: \\'\\',\\n      departmentId: 10619,\\n      departmentName: \\'电商部\\',\\n      projectId: 179018,\\n      projectList: [{\"projectId\":179018,\"projectName\":\"默认分组\",\"smartId\":619,\"brandId\":130259,\"brandName\":\"Semir/森马\",\"bizTypeEnums\":\"STRATEGY\",\"moduleTypeEnum\":\"SAAS\",\"departmentId\":10619,\"departmentName\":\"电商部\",\"departmentTypeEnum\":\"ELEC\",\"projectTypeEnum\":\"DEFAULT\",\"status\":1,\"gmtCreate\":1549093966000}],\\n      feSwitch: {\"offlineOldCategoryAnalysis\":false},\\n      isInternal: false,\\n    };\\n  </script>\\n  <script src=//g.alicdn.com/base-tech-fe/saas-strategy-fe/vendor.js?v=1555809364922 crossorigin=\"anonymous\"></script>\\n  <script src=//g.alicdn.com/base-tech-fe/saas-strategy-fe/nvc_config.js?v=1555809364922 crossorigin=\"anonymous\"></script>\\n  <script type=\"text/javascript\" charset=\"utf-8\" src=//g.alicdn.com/sd/nvc/1.1.112/guide.js?v=1555809364922></script>\\n  <script src=//g.alicdn.com/base-tech-fe/saas-strategy-fe/index.js?v=1555809364922 crossorigin=\"anonymous\"></script>\\n  <script>\\n    try{\\n        window.strategyTopNotice = {\"imgUrl\":\"https://img.alicdn.com/tfs/TB1cAemgAyWBuNjy0FpXXassXXa-14-14.svg\",\"notice\":\"品类偏好分析升级-0404\"}\\n    }catch(e){\\n        console.log(e);\\n    }\\n</script>\\n  <script>\\n    try{\\n        window.strategyNoticePath = {\"noticeUrl\":\"/strategyNoticeTest\"}\\n    }catch(e){\\n        console.log(e);\\n    }\\n</script>\\n</body>\\n</html>\\n'"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "response.text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\r\\n<!DOCTYPE html>\\n<html lang=\"en\">\\n<head>\\n  <meta charset=\"UTF-8\">\\n  <meta name=\"viewport\" content=\"width=device-width, initial-scale=1.0\">\\n  <meta http-equiv=\"X-UA-Compatible\" content=\"ie=edge\">\\n  <link rel=\"shortcut icon\" href=\"//img.alicdn.com/tfs/TB1_5kWnKuSBuNjy1XcXXcYjFXa-32-32.png\">\\n  <meta name=\"_csrf\" token=\"57b7e373e6533\">\\n  <meta name=\"data-spm\" data-spm-protocol=\"i\" content=\"a2o2t\" />\\n  <meta name=\"aplus-ajax\" content=\"chksum\">\\n  <meta name=\"aplus-wating\" content=\"1\">\\n  <title>策略中心</title>\\n</head>\\n<body><script>\\nwith(document)with(body)with(insertBefore(createElement(\"script\"),firstChild))setAttribute(\"exparams\",\"category=&userid=2847965855&aplus&yunid=&57b7e373e6533&trid=0b0f57ec15558185882357821ea0a8&asid=AZ+CwKld6LtcCQgbPQAAAAAp0VZbSp1HPw==\",id=\"tb-beacon-aplus\",src=(location>\"https\"?\"//g\":\"//g\")+\".alicdn.com/alilog/mlog/aplus_v2.js\")\\n</script>\\n\\n  <div id=\"content_container\"></div>\\n  <script>\\n    window.baseConfig = {\\n      nick: \\'森马官方旗舰店:欣月\\',\\n      baseId: \\'4e843e3b-9c61-4064-89d7-2f04d8dfc143\\',\\n      accountName: \\'森马官方旗舰店:欣月\\',\\n      isOwner: false,\\n      smartId: 619,\\n      smartType: 0,\\n      isWLogin: false,\\n      brandId: 130259,\\n      brandName: \\'Semir/森马\\',\\n      logoutUrl: \\'//strategy.tmall.com/signOut\\',\\n      nextBiUrl: \\'//insight-engine.tmall.com/\\',\\n      env: \\'PROD\\',\\n      timestamp: 1555818589273,\\n\\t    policyCenterUrl: \\'http://smart.yushanfang.com/human/crowd-choose#/crowd-analysis\\',\\n      databankUrl: \\'//databank.tmall.com/\\',\\n      databankFactoryUrl: \\'https://paas-databank.tmall.com\\',\\n      goToFactoryUrl: \\'/goToFactory\\',\\n      assetsUrl: \\'//g.alicdn.com/base-tech-fe/saas-strategy-fe/\\',\\n      isDemo: false,\\n      departmentList: [{\"departmentId\":10619,\"departmentName\":\"电商部\",\"ownerAccountName\":null,\"bizType\":\"STRATEGY\",\"projectId\":179018,\"smartId\":619,\"selected\":false}],\\n      currentDepartmentId: 10619,\\n      empId: \\'\\',\\n      lastName: \\'\\',\\n      nickName: \\'\\',\\n      industryList: [],\\n      auth: [{\"code\":\"38\",\"status\":1},{\"code\":\"marketOverview\",\"status\":1},{\"code\":\"marketSeg\",\"status\":1},{\"code\":\"customCrowd\",\"status\":1},{\"code\":\"correlationAnalysis\",\"status\":1},{\"code\":\"consumerPathAnalysis\",\"status\":1},{\"code\":\"categoryAnalysis\",\"status\":1},{\"code\":\"brandGainAndLoss\",\"status\":1},{\"code\":\"penetrationTrack\",\"status\":1},{\"code\":\"productTrack\",\"status\":1},{\"code\":\"partHomeIndex\",\"status\":0},{\"code\":\"partLogo\",\"status\":0},{\"code\":\"brandGainAndLossMarketSeg\",\"status\":1},{\"code\":\"darenOptimal\",\"status\":1},{\"code\":\"znfqLife\",\"status\":0}],\\n      a: \\'\\',\\n      b: \\'\\',\\n      c: \\'\\',\\n      departmentId: 10619,\\n      departmentName: \\'电商部\\',\\n      projectId: 179018,\\n      projectList: [{\"projectId\":179018,\"projectName\":\"默认分组\",\"smartId\":619,\"brandId\":130259,\"brandName\":\"Semir/森马\",\"bizTypeEnums\":\"STRATEGY\",\"moduleTypeEnum\":\"SAAS\",\"departmentId\":10619,\"departmentName\":\"电商部\",\"departmentTypeEnum\":\"ELEC\",\"projectTypeEnum\":\"DEFAULT\",\"status\":1,\"gmtCreate\":1549093966000}],\\n      feSwitch: {\"offlineOldCategoryAnalysis\":false},\\n      isInternal: false,\\n    };\\n  </script>\\n  <script src=//g.alicdn.com/base-tech-fe/saas-strategy-fe/vendor.js?v=1555809364922 crossorigin=\"anonymous\"></script>\\n  <script src=//g.alicdn.com/base-tech-fe/saas-strategy-fe/nvc_config.js?v=1555809364922 crossorigin=\"anonymous\"></script>\\n  <script type=\"text/javascript\" charset=\"utf-8\" src=//g.alicdn.com/sd/nvc/1.1.112/guide.js?v=1555809364922></script>\\n  <script src=//g.alicdn.com/base-tech-fe/saas-strategy-fe/index.js?v=1555809364922 crossorigin=\"anonymous\"></script>\\n  <script>\\n    try{\\n        window.strategyTopNotice = {\"imgUrl\":\"https://img.alicdn.com/tfs/TB1cAemgAyWBuNjy0FpXXassXXa-14-14.svg\",\"notice\":\"品类偏好分析升级-0404\"}\\n    }catch(e){\\n        console.log(e);\\n    }\\n</script>\\n  <script>\\n    try{\\n        window.strategyNoticePath = {\"noticeUrl\":\"/strategyNoticeTest\"}\\n    }catch(e){\\n        console.log(e);\\n    }\\n</script>\\n</body>\\n</html>\\n'"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "response.text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "# iframe2 = driver.find_element_by_css_selector('#sufei-dialog-content')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "no need switch to iframe\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'滑动解锁完成'"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# iframe2 = driver.find_element_by_css_selector('#sufei-dialog-content')\n",
    "def outside_code():\n",
    "    try:\n",
    "        iframe = driver.find_element_by_css_selector('#sufei-dialog-content')\n",
    "        driver.switch_to.frame(iframe)\n",
    "    except:\n",
    "        print('no need switch to iframe')\n",
    "    try:\n",
    "        action = ActionChains(driver)\n",
    "        source=driver.find_element_by_xpath(\"//*[@id='nc_1_n1t']/span\")#需要滑动的元素\n",
    "        action.click_and_hold(source).perform()\n",
    "        time.sleep(0.5)\n",
    "        action.move_by_offset(258,0)#需要滑动的坐标\n",
    "        time.sleep(1.2)\n",
    "        action.release().perform() #释放鼠标\n",
    "    except:\n",
    "        pass\n",
    "    return '滑动解锁完成'\n",
    "outside_code()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.get('https://only.m.tmall.com/shop/shop_auction_search.htm?suid=356060330&sort=default')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "# driver.execute_script(\"window.scrollBy(0, 700)\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "def scroll_phone():\n",
    "    print('---正在滚动，请等待')\n",
    "    for i in range(30):\n",
    "        time.sleep(1)\n",
    "        driver.execute_script(\"window.scrollTo(0, document.body.scrollHeight)\")\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for i in range(10):\n",
    "#     a = i * 10000\n",
    "#     b = (i+1) * 10000\n",
    "#     script = \"window.scrollTo({},{})\".format(a, b)\n",
    "#     driver.execute_script(script)\n",
    "#     time.sleep(random.randint(1, 3) + 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 获取总页数的函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "def func_totalpage():\n",
    "    firt_page_url = 'https://{}.m.tmall.com/shop/shop_auction_search.do?ajson=1&_tm_source=tmallsearch&sort=s&p=1&page_size=12&from=h5&shop_id=57303596&callback=jsonp_{}'.format(shop_name, num)\n",
    "    driver.get(firt_page_url)\n",
    "    html = driver.page_source\n",
    "    try:\n",
    "        infos = re.findall(r'\\(({.*})\\)', html)[0]\n",
    "        infos = json.loads(infos)\n",
    "        totalpage = int(infos.get('total_page'))\n",
    "        return int(totalpage) #*1.04\n",
    "    except:\n",
    "        raise IndexError"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_totalpage(shop_name):\n",
    "    try:\n",
    "        totalpage = func_totalpage()\n",
    "    except IndexError:\n",
    "        driver.get('https://{}.m.tmall.com/shop/shop_auction_search.htm?sort=default'.format(shop_name))\n",
    "        time.sleep(random.random()*5 + 1)\n",
    "        hand = input('\\n总页数，请手动滑动解锁\\n')\n",
    "        if hand == 'b':\n",
    "            raise ValueError\n",
    "        else:\n",
    "            pass\n",
    "#         print(outside_code())\n",
    "        totalpage = func_totalpage()\n",
    "    return totalpage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "def try_totalpage():\n",
    "    try:\n",
    "        totalpage = get_totalpage(shop_name)\n",
    "        return totalpage\n",
    "    except IndexError:\n",
    "        return try_totalpage()\n",
    "    except ValueError:\n",
    "        return 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 获取所有品牌的总宝贝页数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_one_url():\n",
    "    \"\"\"从文件获取需要爬取的url链接\"\"\"\n",
    "    url_file = r\"D:\\Python\\TmallCrawler\\file\\Tmall_shop_home_clothe\"\n",
    "    with open(url_file, 'r', encoding='utf-8') as f:\n",
    "        csv_reader = f.read().split('\\n')\n",
    "        for row in csv_reader:\n",
    "            yield row.split('\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GU is beginning\n",
      "GU have 47 page items\n",
      "家乐芬 is beginning\n",
      "家乐芬 have 34 page items\n",
      "OYSHO is beginning\n",
      "OYSHO have 77 page items\n",
      "安之伴 is beginning\n",
      "安之伴 have 19 page items\n",
      "芬腾 is beginning\n",
      "芬腾 have 42 page items\n",
      "果壳 is beginning\n",
      "果壳 have 29 page items\n",
      "贝妍 is beginning\n",
      "贝妍 have 27 page items\n",
      "优衣库 is beginning\n",
      "优衣库 have 103 page items\n"
     ]
    }
   ],
   "source": [
    "brand_json = []\n",
    "for brand_name, shop_url in get_one_url():\n",
    "    shop_name = shop_url[8:-12]\n",
    "    print('{} is beginning'.format(brand_name))\n",
    "    num = random.randint(10127482, 75727482)\n",
    "    totalpage = try_totalpage()\n",
    "    print('{} have {} page items'.format(brand_name, totalpage))\n",
    "    brand_json.append((brand_name, shop_name, shop_url, totalpage))\n",
    "with open('brand_json', 'wb+') as f:\n",
    "    f.write(pickle.dumps(brand_json))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "navigator set successfully\n"
     ]
    }
   ],
   "source": [
    "set_navigator()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 获取每一页数据，对于获取失败的，保存，最后重新下载"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 重新读取所有的品牌和总页数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('brand_json', 'rb') as f:\n",
    "    brand_json = pickle.loads(f.read())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 获取每页的宝贝列表函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_one_page(shop_name, i, num):\n",
    "    print('---第',i,'页正在加载')\n",
    "    url = 'https://{}.m.tmall.com/shop/shop_auction_search.do?ajson=1&_tm_source=tmallsearch&sort=s&p={}&page_size=12&from=h5&callback=jsonp_{}'.format(shop_name, i, num)\n",
    "    #https://handuyishe.m.tmall.com/shop/shop_auction_search.do?sort=s&p=1&page_size=12&from=h5&ajson=1&_tm_source=tmallsearch&callback=jsonp_42750715\n",
    "    driver.get(url)\n",
    "    time.sleep(random.random() + 1)\n",
    "    html = driver.page_source\n",
    "    if re.findall(r'\\(({.*})\\)', html):\n",
    "        infos = re.findall(r'\\(({.*})\\)', html)[0]\n",
    "        infos = json.loads(infos)\n",
    "        products = infos.get('items')\n",
    "        if products:\n",
    "            for one_item in products:\n",
    "                one_item['store_name'] = shop_name\n",
    "                one_item['page'] = str(i)\n",
    "        else:\n",
    "            raise IndexError\n",
    "    else:\n",
    "        raise IndexError\n",
    "    return products"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 对每个店铺的数据进行保存"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "# store_name = input('请输入店铺名： ')\n",
    "def get_one_store(brand_name, shop_url, begin_page):\n",
    "    shop_name = shop_url[8:-12]\n",
    "    all_page = []\n",
    "    for i in range(begin_page, totalpage+1):\n",
    "        print('当前页面是: 第', i, '页')\n",
    "        try:\n",
    "            products = get_one_page(shop_name, i, num) #获取每页的数据\n",
    "            df = pd.DataFrame(products)\n",
    "            col = df.columns\n",
    "            df.to_csv('D:\\\\Jupyter\\\\all_brand\\\\0110\\\\{}_0110.csv'.format(brand_name), mode='a+', header=False, columns=col, encoding='utf-8')\n",
    "            print('---第 {} 页加载完成\\n'.format(i))\n",
    "        except (TypeError, IndexError):\n",
    "            driver.get('{}/shop/shop_auction_search.htm?sort=default'.format(shop_url))\n",
    "            scroll_phone()\n",
    "            hand = input('\\n单页，请手动滑动解锁\\n')\n",
    "            print(hand)\n",
    "            one_brank = '{},{}\\n'.format(brand_name, i)\n",
    "            with open('break_file', 'a+') as f:\n",
    "                f.write(one_brank)\n",
    "            if hand == '1':\n",
    "                raise IndexError\n",
    "            else:\n",
    "                pass\n",
    "    return '------{} finished!'.format(brand_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "def try_one_store():\n",
    "    try:\n",
    "        print(get_one_store(brand_name, shop_url, begin_page))\n",
    "    except IndexError:\n",
    "         if 'b' == input('need break: '):\n",
    "            return None\n",
    "         else:\n",
    "            return try_one_store()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----GU----: \n",
      "GU is beginning\n",
      "GU have 47 page items\n",
      "起始页数： 1\n",
      "need totalpage: \n",
      "当前页面是: 第 1 页\n",
      "---第 1 页正在加载\n",
      "---第 1 页加载完成\n",
      "\n",
      "当前页面是: 第 2 页\n",
      "---第 2 页正在加载\n",
      "---第 2 页加载完成\n",
      "\n",
      "当前页面是: 第 3 页\n",
      "---第 3 页正在加载\n",
      "---第 3 页加载完成\n",
      "\n",
      "当前页面是: 第 4 页\n",
      "---第 4 页正在加载\n",
      "---第 4 页加载完成\n",
      "\n",
      "当前页面是: 第 5 页\n",
      "---第 5 页正在加载\n",
      "---第 5 页加载完成\n",
      "\n",
      "当前页面是: 第 6 页\n",
      "---第 6 页正在加载\n",
      "---第 6 页加载完成\n",
      "\n",
      "当前页面是: 第 7 页\n",
      "---第 7 页正在加载\n",
      "---第 7 页加载完成\n",
      "\n",
      "当前页面是: 第 8 页\n",
      "---第 8 页正在加载\n",
      "---第 8 页加载完成\n",
      "\n",
      "当前页面是: 第 9 页\n",
      "---第 9 页正在加载\n",
      "---第 9 页加载完成\n",
      "\n",
      "当前页面是: 第 10 页\n",
      "---第 10 页正在加载\n",
      "---第 10 页加载完成\n",
      "\n",
      "当前页面是: 第 11 页\n",
      "---第 11 页正在加载\n",
      "---第 11 页加载完成\n",
      "\n",
      "当前页面是: 第 12 页\n",
      "---第 12 页正在加载\n",
      "---第 12 页加载完成\n",
      "\n",
      "当前页面是: 第 13 页\n",
      "---第 13 页正在加载\n",
      "---第 13 页加载完成\n",
      "\n",
      "当前页面是: 第 14 页\n",
      "---第 14 页正在加载\n",
      "---第 14 页加载完成\n",
      "\n",
      "当前页面是: 第 15 页\n",
      "---第 15 页正在加载\n",
      "---第 15 页加载完成\n",
      "\n",
      "当前页面是: 第 16 页\n",
      "---第 16 页正在加载\n",
      "---第 16 页加载完成\n",
      "\n",
      "当前页面是: 第 17 页\n",
      "---第 17 页正在加载\n",
      "---第 17 页加载完成\n",
      "\n",
      "当前页面是: 第 18 页\n",
      "---第 18 页正在加载\n",
      "---第 18 页加载完成\n",
      "\n",
      "当前页面是: 第 19 页\n",
      "---第 19 页正在加载\n",
      "---第 19 页加载完成\n",
      "\n",
      "当前页面是: 第 20 页\n",
      "---第 20 页正在加载\n",
      "---第 20 页加载完成\n",
      "\n",
      "当前页面是: 第 21 页\n",
      "---第 21 页正在加载\n",
      "---第 21 页加载完成\n",
      "\n",
      "当前页面是: 第 22 页\n",
      "---第 22 页正在加载\n",
      "----正在滚动，请等待\n",
      "\n",
      "单页，请手动滑动解锁\n",
      "\n",
      "\n",
      "当前页面是: 第 23 页\n",
      "---第 23 页正在加载\n",
      "---第 23 页加载完成\n",
      "\n",
      "当前页面是: 第 24 页\n",
      "---第 24 页正在加载\n",
      "---第 24 页加载完成\n",
      "\n",
      "当前页面是: 第 25 页\n",
      "---第 25 页正在加载\n",
      "---第 25 页加载完成\n",
      "\n",
      "当前页面是: 第 26 页\n",
      "---第 26 页正在加载\n",
      "---第 26 页加载完成\n",
      "\n",
      "当前页面是: 第 27 页\n",
      "---第 27 页正在加载\n",
      "---第 27 页加载完成\n",
      "\n",
      "当前页面是: 第 28 页\n",
      "---第 28 页正在加载\n",
      "---第 28 页加载完成\n",
      "\n",
      "当前页面是: 第 29 页\n",
      "---第 29 页正在加载\n",
      "---第 29 页加载完成\n",
      "\n",
      "当前页面是: 第 30 页\n",
      "---第 30 页正在加载\n",
      "---第 30 页加载完成\n",
      "\n",
      "当前页面是: 第 31 页\n",
      "---第 31 页正在加载\n",
      "---第 31 页加载完成\n",
      "\n",
      "当前页面是: 第 32 页\n",
      "---第 32 页正在加载\n",
      "---第 32 页加载完成\n",
      "\n",
      "当前页面是: 第 33 页\n",
      "---第 33 页正在加载\n",
      "---第 33 页加载完成\n",
      "\n",
      "当前页面是: 第 34 页\n",
      "---第 34 页正在加载\n",
      "---第 34 页加载完成\n",
      "\n",
      "当前页面是: 第 35 页\n",
      "---第 35 页正在加载\n",
      "---第 35 页加载完成\n",
      "\n",
      "当前页面是: 第 36 页\n",
      "---第 36 页正在加载\n",
      "---第 36 页加载完成\n",
      "\n",
      "当前页面是: 第 37 页\n",
      "---第 37 页正在加载\n",
      "---第 37 页加载完成\n",
      "\n",
      "当前页面是: 第 38 页\n",
      "---第 38 页正在加载\n",
      "----正在滚动，请等待\n",
      "\n",
      "单页，请手动滑动解锁\n",
      "\n",
      "\n",
      "当前页面是: 第 39 页\n",
      "---第 39 页正在加载\n",
      "---第 39 页加载完成\n",
      "\n",
      "当前页面是: 第 40 页\n",
      "---第 40 页正在加载\n",
      "---第 40 页加载完成\n",
      "\n",
      "当前页面是: 第 41 页\n",
      "---第 41 页正在加载\n",
      "---第 41 页加载完成\n",
      "\n",
      "当前页面是: 第 42 页\n",
      "---第 42 页正在加载\n",
      "---第 42 页加载完成\n",
      "\n",
      "当前页面是: 第 43 页\n",
      "---第 43 页正在加载\n",
      "---第 43 页加载完成\n",
      "\n",
      "当前页面是: 第 44 页\n",
      "---第 44 页正在加载\n",
      "---第 44 页加载完成\n",
      "\n",
      "当前页面是: 第 45 页\n",
      "---第 45 页正在加载\n",
      "---第 45 页加载完成\n",
      "\n",
      "当前页面是: 第 46 页\n",
      "---第 46 页正在加载\n",
      "---第 46 页加载完成\n",
      "\n",
      "当前页面是: 第 47 页\n",
      "---第 47 页正在加载\n",
      "----正在滚动，请等待\n",
      "\n",
      "单页，请手动滑动解锁\n",
      "\n",
      "\n",
      "------GU finished!\n",
      "None\n",
      "----家乐芬----: y\n",
      "家乐芬 is finished\n"
     ]
    }
   ],
   "source": [
    "for one_brand in brand_json[:2]:\n",
    "    brand_name, shop_name, shop_url, totalpage = one_brand\n",
    "    a = input('----{}----: '.format(brand_name))\n",
    "    if a == 'y':\n",
    "        print(brand_name, 'is finished')\n",
    "    else:\n",
    "        print('{} is beginning'.format(brand_name))\n",
    "        num = random.randint(10127482, 85727482)\n",
    "        print('{} have {} page items'.format(brand_name, totalpage))\n",
    "        begin_page = int(input(\"起始页数： \"))\n",
    "        a = input('need totalpage: ')\n",
    "        if a == 'need':\n",
    "            totalpage = int(input('---: '))\n",
    "            print(try_one_store())\n",
    "        else:\n",
    "            print(try_one_store())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
